/**
 * 
 */
package com.gxljc.bear.crawler.itaogao.huanqiu;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;

import com.alibaba.fastjson.JSONObject;
import com.gxljc.commons.util.Pair;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.itaogao.NewsPageTable;
import com.gxljc.bear.crawler.itaogao.util.FetchDataUtil;
import com.gxljc.bear.crawler.itaogao.util.HbaseUtil;
import com.gxljc.bear.crawler.itaogao.util.HttpUtil;
import com.gxljc.bear.crawler.itaogao.util.ImageUtil;
import com.gxljc.bear.crawler.itaogao.util.MongodbUtil;
import com.gxljc.bear.crawler.util.DateUtil;

/**
 * @author haitao E-mail:haitao@summba.com
 * @version createTime：2021年9月22日 下午6:18:47
 * 
 */

public class HuanqiuCrawler {
	private final static List<String> crawlerSite = new ArrayList<String>();
	private final static String MONGONAME_COLLECTION = "t_site_huanqiu";
	private final static Logger LOG = Logger.getLogger(HuanqiuCrawler.class);
	private Map<String, Object> newsMap = new HashMap<String, Object>();
	private List<String> titleSeeds = new ArrayList<String>();
	private final static String SOURCE = "环球网";
	private static int countCrawler =0;
	protected boolean proxy;
	private int pageNum = 1;

	public HuanqiuCrawler() {
		crawlerSite.add("http://oversea.huanqiu.com/article/");// 海外看中国
		crawlerSite.add("http://world.huanqiu.com/exclusive/");// 国际
		crawlerSite.add("http://opinion.huanqiu.com/roll.html");// 评论roll_2.html
		crawlerSite.add("http://mil.huanqiu.com/china/");// 中国军事
		crawlerSite.add("http://mil.huanqiu.com/world/");// 国际军事
		crawlerSite.add("http://mil.huanqiu.com/observation/");// 军事评论
		proxy = false;
		newsMap.put("status", 0);
		newsMap.put("mediaName", SOURCE);
	}

	/**
	 * =========爬虫启动入口(All)==========
	 */
	public void startCrawlerAll() {
		genDefaultSeeds();
		LOG.info("Start crawler detail huanqiu news!");
		crawlerNews();
		LOG.info(String.format("huanqiu news crawler finished! and cralwer all size is %s", countCrawler));
	}

	// 获取种子列表
	void genDefaultSeeds() {
		for (String seed : crawlerSite) {
			// 判别网址是否存在
			if (!HttpUtil.checkUrlexists(seed)) {
				continue;
			}
			crawlerSeedsFirstPage(seed, "titleUrl");
			if (pageNum > 1) {
				for (int i = 2; i <= pageNum; i++) {
					String htmlNum = String.valueOf(i) + ".html";
					String url = "";
					if (seed.contains("roll.html")) {
						url = seed.replace("roll.html", "roll_" + htmlNum);
					} else {
						url = seed + htmlNum;
					}
					// 判别网址存在
					if (!HttpUtil.checkUrlexists(url)) {
						continue;
					}
					crawlerSeedsAllPage(url, "titleUrl");
				}
			}
			if (titleSeeds.size() % 500 == 0) {
				LOG.info(String.format("titleSeeds size is %s", titleSeeds.size()));
			}
		}
		LOG.info(String.format("DefaultSeeds crawler finished,all size is %s", titleSeeds.size()));
	}

	@SuppressWarnings("unchecked")
	void crawlerSeedsFirstPage(String url, String key) {
		Pair<String, byte[]> pair = FetchDataUtil.fetResult(url, proxy);
		if (pair != null) {
			String result = "";
			result = pair.first;
			if (result != null) {
				JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
				String pNum = ObjectUtils.toString(jsonResult.get("pageNum"));
				if (!pNum.isEmpty()) {
					pageNum = Integer.parseInt(pNum);
				}
				if (jsonResult != null && jsonResult.get(key) != null) {
					for (String titleUrl : (List<String>) jsonResult.get(key)) {
						if (!MongodbUtil.needReCrawl(titleUrl, MONGONAME_COLLECTION)) {
							titleSeeds.add(titleUrl);
						}
					}

				}
			}

		}
	}

	@SuppressWarnings("unchecked")
	void crawlerSeedsAllPage(String url, String key) {
		Pair<String, byte[]> pair = FetchDataUtil.fetResult(url, proxy);
		if (pair != null) {
			String result = "";
			result = pair.first;
			if (result != null) {
				JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
				if (jsonResult != null && jsonResult.get(key) != null) {
					for (String titleUrl : (List<String>) jsonResult.get(key)) {
						if (!MongodbUtil.needReCrawl(titleUrl, MONGONAME_COLLECTION)) {
							titleSeeds.add(titleUrl);
						}
					}

				}
			}

		}
	}

	/**
	 * 抓取新闻详情
	 */
	@SuppressWarnings("unchecked")
	void crawlerNews() {
		if (titleSeeds.size() == 0) {
			LOG.info(String.format("titleSeeds Size is %s", titleSeeds.size()));
			return;
		}
		for (String seed : titleSeeds) {
			LOG.info(String.format("crawlering seed is %s", seed));
			Pair<String, byte[]> pair = null;
			for (int i = 0; i <= 4; i++) {
				pair = FetchDataUtil.fetResult(seed, proxy);
				if (pair != null) {
					break;
				}
			}
			if (pair == null) {
				LOG.info(String.format("fetResult failed!seed is %s", seed));
				continue;
			}
			String result = "";
			result = pair.first;
			byte[] html = pair.second;
			NewsPageTable table = new NewsPageTable();
			if (result != null) {
				JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
				List<String> contentList = new ArrayList<String>();
				Object content = jsonResult.get("content");
				if (content != null) {
					contentList = (List<String>) content;
				}

				// 基础数据
				String id = ObjectUtils.toString(StringUtil.genMD5Val(seed));
				String title = ObjectUtils.toString(jsonResult.get("title"));
				String subTitle = ObjectUtils.toString(jsonResult.get("subTitle"));
				String logoUrl = ObjectUtils.toString(jsonResult.get("logoUrl"));
				String publishDate = ObjectUtils.toString(jsonResult.get("publicTime"));
				String source = ObjectUtils.toString(jsonResult.get("source"));

				// 部分需要处理数据
				Object author = jsonResult.get("author");
				List<String> authorList = new ArrayList<String>();
				if (author != null) {
					authorList = (List<String>) author;
				}
				if (authorList.size() != 0) {
					newsMap.put("author", StringUtils.join(authorList, ","));
				}

				if (!logoUrl.isEmpty()) {
					try {
						// 补齐相对图片路径
						logoUrl = ImageUtil.imageUrlSupply(seed, logoUrl);
						// 图片平台转换
						logoUrl = ImageUtil.imageConvertplatform(logoUrl);
					} catch (Exception e) {
						LOG.info(String.format("logoUrl convert failed! logoUrl is %s", logoUrl));
						e.printStackTrace();
					}
				}

				// 若logoUrl不为空才有images
				Object images = jsonResult.get("images");
				List<String> imagesNew = new ArrayList<String>();
				if (images != null && !logoUrl.isEmpty()) {
					imagesNew = ImageUtil.ImageUrlService((List<String>) images, seed);
				}
				if (imagesNew.size() != 0) {
					newsMap.put("images", imagesNew);
				}

				Date pd = DateUtil.parseFormat(publishDate, "yyyy-MM-dd HH:mm:ss");
				if (pd != null) {
					newsMap.put("publishDate", pd);
				}
				// 添加基础数据
				newsMap.put("_id", id);
				newsMap.put("title", title);
				newsMap.put("subTitle", subTitle);
				newsMap.put("logoUrl", logoUrl);
				if (contentList.size() != 0) {
					newsMap.put("content", StringUtils.join(contentList, "####").replaceAll("　　####|　　", "").trim());
				}
				newsMap.put("crawlUrl", seed);
				newsMap.put("crawlFlag", 1);
				newsMap.put("crawlDate", new Date());
				newsMap.put("source", source);

				// 更新mongdb
				MongodbUtil.update2Mongodb(newsMap, MONGONAME_COLLECTION);

				table.setTitle(title);
				table.setId(id);

				// 更新到hbase
				HbaseUtil.saveHbase(table, html);
				countCrawler+=1;
				LOG.info(String.format("insert the id/rowkey is %s ", id));
			}

		}
	}

	@SuppressWarnings("unchecked")
	void crawlerNewsSingle(String seed) {
		Pair<String, byte[]> pair =null;
		for(int i=0;i<=4;i++){
			pair = FetchDataUtil.fetResult(seed, proxy);
			if(pair!=null){
				break;
			}
		}
		if (pair == null) {
			LOG.info(String.format("fetResult failed!seed is %s", seed));
			return;
		}

		String result = "";
		result = pair.first;
		byte[] html = pair.second;
		NewsPageTable table = new NewsPageTable();
		if (result != null) {
			JSONObject jsonResult = (JSONObject) JSONObject.parse(result);
			List<String> contentList = new ArrayList<String>();
			Object content = jsonResult.get("content");
			if (content != null) {
				contentList = (List<String>) content;
			}

			// 基础数据
			String id = ObjectUtils.toString(StringUtil.genMD5Val(seed));
			String title = ObjectUtils.toString(jsonResult.get("title"));
			String subTitle = ObjectUtils.toString(jsonResult.get("subTitle"));
			String logoUrl = ObjectUtils.toString(jsonResult.get("logoUrl"));
			String source = ObjectUtils.toString(jsonResult.get("source"));

			// 部分需要处理数据
			Object author = jsonResult.get("author");
			List<String> authorList = new ArrayList<String>();
			if (author != null) {
				authorList = (List<String>) author;
			}
			if (authorList.size() != 0) {
				newsMap.put("author", StringUtils.join(authorList, ","));
			}
			
			// 若logoUrl不为空才有images
			if (!logoUrl.isEmpty()) {
				try {
					// 补齐相对图片路径
					logoUrl = ImageUtil.imageUrlSupply(seed, logoUrl);
					// 图片平台转换
					logoUrl = ImageUtil.imageConvertplatform(logoUrl);
				} catch (Exception e) {
					LOG.info(String.format("logoUrl convert failed! logoUrl is %s", logoUrl));
					e.printStackTrace();
				}
			}

			Object images = jsonResult.get("images");
			List<String> imagesNew = new ArrayList<String>();
			if (images != null && !logoUrl.isEmpty()) {
				imagesNew = ImageUtil.ImageUrlService((List<String>) images, seed);
			}
			if (imagesNew.size() != 0) {
				newsMap.put("images", imagesNew);
			}

			String publishDate = ObjectUtils.toString(jsonResult.get("publicTime"));
			Date pd = DateUtil.parseFormat(publishDate, "yyyy-MM-dd HH:mm:ss");
			if (pd != null) {
				newsMap.put("publishDate", pd);
			}
			// 添加基础数据
			newsMap.put("_id", id);
			newsMap.put("title", title);
			newsMap.put("subTitle", subTitle);
			newsMap.put("logoUrl", logoUrl);
			if (contentList.size() != 0) {
				newsMap.put("content", StringUtils.join(contentList, "####").replaceAll("　　####|　　", "").trim());
			}
			newsMap.put("crawlUrl", seed);
			newsMap.put("crawlFlag", 1);
			newsMap.put("crawlDate", new Date());
			newsMap.put("source", source);

			// 更新mongdb
			MongodbUtil.update2Mongodb(newsMap, MONGONAME_COLLECTION);

			table.setTitle(title);
			table.setId(id);

			// 更新到hbase
			HbaseUtil.saveHbase(table, html);

			LOG.info(String.format("insert the id/rowkey is %s ", id));
		}

	}

}
